package ca.uwindsor.cs.deepweb.utility;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.LockObtainFailedException;

import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.estimation.experiment.RandomFilesContentIndexer;

public class DoubleSelectionRandomFilesContentIndexer extends
		RandomFilesContentIndexer {

	protected Set<String> set_fatherdocumentname;

	protected Set<Integer> set_fatherdocumentid;

	protected File[] dataFiles;

	public DoubleSelectionRandomFilesContentIndexer(String sourcepath,
			String indexdir, int amount) {
		super(sourcepath, indexdir, amount);
		// TODO Auto-generated constructor stub
		set_fatherdocumentname = new HashSet<String>();
		set_fatherdocumentid = new HashSet<Integer>();
		dataFiles = dataDir.listFiles();
	}

	/**
	 * Start to index the files in the specified directory. One file's content
	 * is an instance of Document in Lucene.
	 */
	public void generateIndex() {
		try {
			Analyzer luceneAnalyzer = new StandardAnalyzer();
			IndexWriter indexWriter = new IndexWriter(indexDir, luceneAnalyzer,
					true);

			length = dataFiles.length;
			long startTime = new Date().getTime();
			long id = 0;
			int i;
			while (id < quality) {
				i = getRandomOffset();
				if (dataFiles[i].isFile()
						&& dataFiles[i].getName().endsWith(".txt")) {

					Document document = new Document();
					InputStreamReader txtReader = new InputStreamReader(
							new FileInputStream(dataFiles[i]), "UTF-8");

					String word = String.valueOf(i);

					Field FieldPath = new Field(FilesContentIndexer.FIELD_ID,
							word, Field.Store.YES, Field.Index.UN_TOKENIZED);
					Field FieldBody = new Field(
							FilesContentIndexer.FIELD_CONTENT, txtReader);

					document.add(FieldPath);
					document.add(FieldBody);
					indexWriter.addDocument(document);

					set_fatherdocumentid.add(i);
					set_fatherdocumentname.add(dataFiles[i].getName());

					id++;
				}
			}
			indexWriter.close();
			long endTime = new Date().getTime();

			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ dataDir.getPath()
							+ "\nTotal documnents indexed is:" + id);
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	public void generateIndex(int size, String indexpath) {
		try {
			Analyzer luceneAnalyzer = new StandardAnalyzer();
			IndexWriter indexWriter = new IndexWriter(indexpath,
					luceneAnalyzer, true);

			length = dataFiles.length;
			long startTime = new Date().getTime();
			long id = 0;
			int i;
			Set<Integer> set = new HashSet<Integer>();
			while (id < size) {
				i = getRandomOffset(set);

				Document document = new Document();
				InputStreamReader txtReader = new InputStreamReader(
						new FileInputStream(dataFiles[i]), "UTF-8");

				String word = String.valueOf(i);

				Field FieldPath = new Field(FilesContentIndexer.FIELD_ID, word,
						Field.Store.YES, Field.Index.UN_TOKENIZED);
				Field FieldBody = new Field(FilesContentIndexer.FIELD_CONTENT,
						txtReader);

				document.add(FieldPath);
				document.add(FieldBody);
				indexWriter.addDocument(document);

				set.add(i);

				id++;
			}
			indexWriter.close();
			long endTime = new Date().getTime();

			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ dataDir.getPath()
							+ "\nTotal documnents indexed is:" + id);
		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}
	}

	protected int getRandomOffset(Set<Integer> set_base) {
		int index = 0;
		do {
			double random = ((length - 1) * Math.random());
			BigDecimal i = new BigDecimal(random).setScale(0,
					BigDecimal.ROUND_HALF_UP);
			index = Integer.parseInt(i.toString());
		} while (!set_fatherdocumentid.contains(index));
		
		while (set_base.contains(index)){
			double random = ((length - 1) * Math.random());
			BigDecimal i = new BigDecimal(random).setScale(0,
					BigDecimal.ROUND_HALF_UP);
			index = Integer.parseInt(i.toString());
			if (!set_fatherdocumentid.contains(index)) {
				continue;
			}
		}
		set_base.add(index);
		return index;
	}

	public Set<String> getFatherDcouments() {
		return set_fatherdocumentname;
	}

	public static void main(String[] args) {
		// args = new String[3];
		// args[0] = "SourceData\\exp";
		// args[1] = "Experiment\\exp";
		// args[2] = "2";
		if (args.length != 8) {
			System.out
					.println("usage: java -jar DoubleSelectionRandomFilesContentIndexer.jar sourcefiledirectory indexdirectory quality dir_for_2k dir_for_3k dir_for_4k dir_for_5k file_store_indexfiles!");
		} else {
			DoubleSelectionRandomFilesContentIndexer index = new DoubleSelectionRandomFilesContentIndexer(
					args[0], args[1], Integer.parseInt(args[2]));
			System.out.println("Start to Index!");
			System.out.println("Indexing from "+ args[0] + " to " + args[1] + "...");
			index.generateIndex();
			System.out.println("Indexing is completed!");
			Set<String> set = index.getFatherDcouments();
			System.out.println("Documents are recorded!");
			try {
				FileOutputStream fp = new FileOutputStream(args[7]);
				for (String s : set) {
					String name = s + "\n";
					fp.write(name.getBytes("UTF-8"));
				}
				fp.close();
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				System.exit(1);
			} catch (UnsupportedEncodingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				System.exit(1);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				System.exit(1);
			}
			System.out.println("Indexing 2k "+ " to " + args[3] + "...");
			index.generateIndex(2000, args[3]);
			System.out.println("Indexing is completed!");
			System.out.println("Indexing 3k "+ " to " + args[4] + "...");
			index.generateIndex(3000, args[4]);
			System.out.println("Indexing is completed!");
			System.out.println("Indexing 4k "+ " to " + args[5] + "...");
			index.generateIndex(4000, args[5]);
			System.out.println("Indexing is completed!");
			System.out.println("Indexing 5k "+ " to " + args[6] + "...");
			index.generateIndex(5000, args[6]);
			System.out.println("Indexing is completed!");
		}
		
	}
}
